package TFIDF;


import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.wltea.analyzer.core.IKSegmenter;
import org.wltea.analyzer.core.Lexeme;

import java.io.IOException;
import java.io.StringReader;

/**
 * 接收到的数据
 * 肖申克的救赎   希望让人自由
 * 输出  <count,1>    <希望_肖申克的救赎,1>
 */
public class MapTest01 extends Mapper<LongWritable, receiveData, Text, DoubleWritable> {
    DoubleWritable v = new DoubleWritable(1);

    @Override
    protected void map(LongWritable key, receiveData value, Context context) throws IOException, InterruptedException {
        String datas[] = value.toString().split("\t");
        if ("none".equalsIgnoreCase(datas[1]))return;
        StringReader sr = new StringReader(datas[1]);
        StringReader sr1 = new StringReader(datas[1]);
        //统计每个分词的数量
        int count = 0;
        IKSegmenter ik1 = new IKSegmenter(sr1, true);
        Lexeme l = null;
        while ((l = ik1.next()) != null) {
            count++;
        }
        //将评论进行分词
        IKSegmenter ik = new IKSegmenter(sr, true);
        Lexeme lex = null;
        while ((lex = ik.next()) != null) {
            //输出k是分词加上电影名，v是1
            context.write(new Text(lex.getLexemeText() + "_" + datas[0] + "_" + count), v);
        }
        context.write(new Text("count"), v);
    }
}
